package com.darrenchan.spark.javaapi;

import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.SparkSession;
import scala.Tuple2;

import java.util.Arrays;

/**
 * 文件内容：
 * hello spark
 * hello hadoop
 * hello flink
 * hello storm
 */
public class WordCountApp2 {
    public static void main(String[] args) {
        SparkConf sparkConf = new SparkConf().setMaster("local[2]").setAppName("WordCountApp");
        JavaSparkContext sc = new JavaSparkContext(sparkConf);

        //业务逻辑
        JavaPairRDD<String, Integer> counts =
                sc.textFile("D:\\hello.txt").
                        flatMap(line -> Arrays.asList(line.split(" ")).iterator()).
                        mapToPair(word -> new Tuple2<>(word, 1)).
                        reduceByKey((a, b) -> a + b);

        System.out.println(counts.collect());

        sc.stop();
    }
}
